#!/usr/bin/python3
#
# Mirror the iLO firmware directory and create firmware.conf for the hpilo
# repository. Not meant to be used by end users.
#
# (c) 2011-2020 Dennis Kaarsemaker <dennis@kaarsemaker.net>
# see COPYING for license details

import configparser
import ftplib
import glob
import io
import optparse
import os
import re
import requests
import sys
import tarfile
from whelk import Shell
try:
    git = Shell(redirect=False, encoding='utf-8').git
except AttributeError:
    git = lambda *args: True

def main():
    p = optparse.OptionParser()
    p.add_option('-v', '--verbose', dest="verbose", action="store_true", default=False,
                 help="Be more verbose")
    p.add_option('-q', '--quiet', dest="quiet", action="count", default=0,
                 help="no output except errors and updates. Repeat to get only errors")
    p.add_option('--blank-files', dest='blank_files', action="store_true", default=False,
                 help="Delete downloaded content after processing")
    p.add_option('--reset', dest='reset', action="store_true", default=False,
                 help="Reset git tree to origin/master before mirroring")
    p.add_option('--fetch', dest='fetch', action="store_true", default=False,
                 help="Fetch git data before mirroring")
    p.add_option('--keep', dest='keep', action="store_true", default=False,
                 help="Keep changes")
    opts, args = p.parse_args()

    repo = os.path.abspath(os.path.dirname(__file__))
    repo = git('-C', repo, 'rev-parse', '--show-toplevel', stdout=Shell.PIPE).stdout.strip()
    os.chdir(repo)

    # Do we need to update?
    if opts.fetch:
        git('fetch', '--quiet')

    if opts.reset:
        git('reset', '--quiet', '--hard', 'origin/master')

    # Abort if there are local changes
    if not git('diff', '--quiet', '--exit-code', '--', 'firmware.conf'):
        print("Local changes to firmware.conf, aborting")
        return

    # Mirror all files and write config
    ftproot = ('ftp.hp.com', '/pub/softlib2/software1/sc-linux-fw-ilo')
    conf = os.path.join(repo, 'firmware.conf')
    root = os.path.join(repo, 'devtools', 'firmware')

    ftp = FTP(ftproot[0], conf=conf, verbosity = 0 if opts.quiet else 2 if opts.verbose else 1, blank_files=opts.blank_files)

    # ftp.login('anonymous','anonymous')
    # ftp.mirror(ftproot[1], root)
    resp = requests.get('http://pingtool.org/latest-hp-ilo-firmwares/')
    ftp.mirror_urls(re.findall('http://downloads.hpe.com[^ "<]*', resp.text), root)

    os.chdir(repo)
    result = git('diff', '--quiet', '--exit-code', '--', 'firmware.conf')
    if not result:
        git('--no-pager', 'diff', '--color=never', '--', 'firmware.conf')
        if not opts.keep:
            git('checkout', '--', 'firmware.conf')

class FTP(ftplib.FTP):
    def __init__(self, *args, **kwargs):
        self.host = args[0]
        self.conf = kwargs.pop('conf', None)
        self.verbosity = kwargs.pop('verbosity', None)
        self.blank_files = kwargs.pop('blank_files', None)
        # super(FTP, self).__init__(*args, **kwargs)
        self.configparser = configparser.ConfigParser()
        self.configparser.read(self.conf)

    def log(self, level, msg):
        if self.verbosity >= level:
            print(msg)

    def mirror(self, remote, local):
        if not os.path.exists(local):
            os.mkdir(local)
        os.chdir(local)
        self.cwd(remote)

        lines = []
        self.retrlines('LIST', lines.append)
        for line in lines:
            if line.startswith('total'):
                continue
            mode, _, _, _, _, _, _, _, name = line.split(None, 8)
            if name in ('.', '..'):
                continue
            if mode.startswith('d'):
                self.log(1, "Mirroring %s" % name)
                if os.path.exists(name) and name.startswith('v'):
                    os.chdir(name)
                    if glob.glob('*scexe'):
                        scexe = glob.glob('*.scexe')[0]
                        self.extract(scexe)
                        bin = glob.glob('*.bin')[0]
                        self.add_version("%s/%s" % (remote, name), scexe, bin)
                    if self.blank_files:
                        for file in os.listdir('.'):
                            if os.path.isfile(file):
                                open(file, 'w').close()
                    os.chdir('..')
                    continue
                self.mirror(os.path.join(remote, name), os.path.join(local, name))
                if self.blank_files:
                    for file in os.listdir('.'):
                        if os.path.isfile(file):
                            open(file, 'w').close()
                self.cwd('..')
                os.chdir('..')
            else:
                if not os.path.exists(name):
                    self.log(1, "Downloading %s" % name)
                    with open(name, 'wb') as fd:
                        self.retrbinary('RETR %s' % name, fd.write)
                if name.endswith('.scexe'):
                    self.extract(name)
                    bin = glob.glob('*.bin')[0]
                    self.add_version(remote, name, bin)
        for k in sorted(self.configparser._sections.keys()):
            self.configparser._sections.move_to_end(k)
        with open(self.conf, 'w') as fd:
            self.configparser.write(fd)

    def mirror_urls(self, urls, local):
        if not os.path.exists(local):
            os.mkdir(local)
        os.chdir(local)

        for url in urls:
            path = url.split('/')[-3:]
            dn = os.sep.join(path[-3:-1])
            if not os.path.exists(dn):
                os.makedirs(dn)
            fn = os.sep.join(path[-3:])
            if not os.path.exists(fn):
                self.log(1, "Downloading %s" % url)
                resp = requests.get(url, stream=True)
                if resp.status_code != 200:
                    continue
                with open(fn, 'wb') as fd:
                    for chunk in resp.iter_content(chunk_size=4096):
                        fd.write(chunk)
            os.chdir(dn)

            if glob.glob('*scexe'):
                scexe = glob.glob('*.scexe')[0]
                self.extract(scexe)
                bin = glob.glob('*.bin')[0]
                self.add_version(url, scexe, bin)

            if self.blank_files:
                for file in os.listdir('.'):
                    if os.path.isfile(file):
                        open(file, 'w').close()
                if self.blank_files:
                    for file in os.listdir('.'):
                        if os.path.isfile(file):
                            open(file, 'w').close()

            os.chdir(local)

        for k in sorted(self.configparser._sections.keys()):
            self.configparser._sections.move_to_end(k)
        with open(self.conf, 'w') as fd:
            self.configparser.write(fd)

    def extract(self, name):
        if glob.glob('*.xml'):
            return
        self.log(2, "Processing %s" % name)
        with open(name, 'rb') as fd:
            scexe = fd.read()
        # An scexe is a shell script with an embedded compressed tarball. Find the tarball.
        skip_start = scexe.index(b'_SKIP=') + 6
        skip_end = scexe.index(b'\n', skip_start)
        skip = int(scexe[skip_start:skip_end]) - 1
        tarball = scexe.split(b'\n', skip)[-1]

        # Now uncompress it
        if tarball[:2] != b'\x1f\x8b':
            raise ValueError("Downloaded scexe file %s seems corrupt" % name)

        tf = tarfile.open(fileobj=io.BytesIO(tarball), mode='r:gz')
        for bf in tf.getnames():
            self.log(2, " Extracting %s" % bf)
            tf.extract(bf, '.')

    def add_version(self, remote, scexe, bin):
        bin = bin.lower()
        url = remote if remote.startswith('http://') else 'http://%s%s/%s' % (self.host, remote, scexe)
        if '_' in bin:
            ilo, version = os.path.splitext(bin)[0].split('_')
        else:
            ilo, version = 'ilo', os.path.splitext(bin)[0].replace('ilo', '')
        version = '%s.%s' % (version[0], version[1:])
        if version.endswith('j'):
            return
        vsection = '%s %s' % (ilo, version)
        if not self.configparser.has_section(vsection):
            self.configparser.add_section(vsection)
            self.configparser.set(vsection, 'version', version)
            self.configparser.set(vsection, 'url', url)
            self.configparser.set(vsection, 'file', bin)
        if not self.configparser.has_section(ilo):
            self.configparser.add_section(ilo)
        if self.configparser.has_option(ilo, 'version') and version <= self.configparser.get(ilo, 'version'):
            return
        self.log(1, "New %s firmware version: %s (%s)" % (ilo, version, url))
        self.configparser.set(ilo, 'version', version)
        self.configparser.set(ilo, 'url', url)
        self.configparser.set(ilo, 'file', bin)

main()
